# Updated by Wei on 2014/10/07 at school
# support both gov2/clueweb09B
# Updated by Wei on 2014/06/12 at school
# attempt to fit for the clueweb09B dataset
# Updated by Wei 20140518 at school
# build the 2D numerator table based on the (relRank,listLength)
from __future__ import division
from operator import itemgetter, attrgetter

# import gc
import math
import matplotlib
import os
# import pylab
import random
import sys
import time
from sets import Set

# Updated by Wei on 20140518 at school
# This function doesn't seem to be useful any more.
def step0():
    termWithClassLabelBasedOnListLengthDict = {}
    inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relrankWithLengthAnalyze20140102Night/version1/termPieceInfoForQueryTerms"
    print "inputFileName:",inputFileName
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentTerm = lineElements[0]
        numOfRanges = int( lineElements[3] )
        if currentTerm not in termWithClassLabelBasedOnListLengthDict:
            termWithClassLabelBasedOnListLengthDict[currentTerm] = numOfRanges
    print "len(termWithClassLabelBasedOnListLengthDict): ",len(termWithClassLabelBasedOnListLengthDict)
    print "termWithClassLabelBasedOnListLengthDict['0']: ",termWithClassLabelBasedOnListLengthDict['0']
    print "termWithClassLabelBasedOnListLengthDict['00']: ",termWithClassLabelBasedOnListLengthDict['00']
    inputFileHandler.close()
    
    
    
    inputFileName1 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/2014JanPruningResults/ANDSemantics/TOP10PostingsWithTheirRanksFrom95KTrainingQueries"
    inputFileHandler = open(inputFileName1,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentTerm = lineElements[0]
        currentDocID = lineElements[1]
        currentRankInList = float(lineElements[2])
        currentListOfLength = float(lineElements[3])
        currentRelRank = currentRankInList / currentListOfLength
        #if currentTerm not in termWithClassLabelBasedOnListLengthDict:
        #    print "currentTerm: ",currentTerm
        #    print "Error"
        #    exit(1)
    inputFileHandler.close()

# Updated by Wei on 2014/06/13 at school
# Updated by Wei on 2014/05/18 at school
# Try the following function for the stepGap (1)1/3 and (2)1/2
# Compute the numerator table
def step1_build2DNumeratorTable():
    termWithTheirLengthOfListDict = {}
    termWithTheirNumOfPieceDict = {}
    termWithTheirClassLabelBasedOnListLength = {}
    classLabelWithTheirNumOfPiecesDict = {}
    '''
    # stepGap to be 1/3
    # on dodo
    inputAuxFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/predefinedClassLabelLowerBoundBasedOnListLength_numOfPieces_stepGap_one3rd_20140118Night"
    inputAuxFileHandler = open(inputAuxFileName0,"r")
    for line in inputAuxFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentClassLabel = int( lineElements[0] )
        numOfPieces = int(lineElements[2])
        if currentClassLabel not in classLabelWithTheirNumOfPiecesDict:
             classLabelWithTheirNumOfPiecesDict[currentClassLabel] = numOfPieces
    inputAuxFileHandler.close()
    print "len(classLabelWithTheirNumOfPiecesDict): ",len(classLabelWithTheirNumOfPiecesDict)
    print "classLabelWithTheirNumOfPiecesDict[3]: ",classLabelWithTheirNumOfPiecesDict[3]
    '''

    # stepGap to be 1/2
    # on moa:
    # inputAuxFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/secondFactorProbability/predefinedClassLabelUpperBoundOfRelRank20140102Night"
    # on vidaserver1:
    # gov2
    inputAuxFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/predefinedClassLabelUpperBoundOfRelRank20140102Night"
    # clueweb09B
    # inputAuxFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/clueweb09B/predefinedClassLabelLowerBoundOfListLength_clueweb09B_20140612"
    print "inputAuxFileName0:",inputAuxFileName0
    inputAuxFileHandler = open(inputAuxFileName0,"r")
    for line in inputAuxFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentClassLabel = int( lineElements[0] )
        # for gov2 format
        numOfPieces = int(lineElements[1])
        # for clueweb09B format
        # numOfPieces = int(lineElements[2])
        if currentClassLabel not in classLabelWithTheirNumOfPiecesDict:
             classLabelWithTheirNumOfPiecesDict[currentClassLabel] = numOfPieces
    inputAuxFileHandler.close()
    print "len(classLabelWithTheirNumOfPiecesDict): ",len(classLabelWithTheirNumOfPiecesDict)
    print "classLabelWithTheirNumOfPiecesDict[3]: ",classLabelWithTheirNumOfPiecesDict[3]

    # stepGap to be 1/2
    # for gov2
    # inputAuxFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/secondFactorProbability/termPieceInfoForQueryTerms_stepGap_2_OLD"
    # on vidaserver1
    # for gov2
    inputAuxFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/termPieceInfoForQueryTerms_stepGap_2_OLD"
    # for clueweb09B
    # inputAuxFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/clueweb09B/termPieceInfo_trainingQueryTerms_clueweb09B_20140613"
    inputAuxFileHandler = open(inputAuxFileName1,"r")
    
    for line in inputAuxFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # for gov2 format
        base = 0
        # for clueweb09B format
        # base = 1
        currentTerm = lineElements[base + 0]
        lengthOfTheList = int(lineElements[base + 1])
        classLabelBasedOnListLength = int(lineElements[base + 2])
        numOfPieces = int(lineElements[base + 3])
        # print currentTerm,lengthOfTheList,classLabelBasedOnListLength,numOfPieces
        if currentTerm not in termWithTheirLengthOfListDict:
            termWithTheirLengthOfListDict[currentTerm] = lengthOfTheList
            termWithTheirClassLabelBasedOnListLength[currentTerm] = classLabelBasedOnListLength
            termWithTheirNumOfPieceDict[currentTerm] = numOfPieces
    inputAuxFileHandler.close()
    print "len(termWithTheirLengthOfListDict): ",len(termWithTheirLengthOfListDict)
    print "termWithTheirLengthOfListDict['fawn']: ",termWithTheirLengthOfListDict['fawn']
    print "len(termWithTheirClassLabelBasedOnListLength): ",len(termWithTheirClassLabelBasedOnListLength)
    print "len(termWithTheirNumOfPieceDict): ",len(termWithTheirNumOfPieceDict)

    # key: term
    # value: a list containing all the postings
    termPostingListDict = {}
    # for gov2
    # the formats for the below two files are somehow different
    # on dodo:
    # column0: query term
    # column1: docID
    # column2: rank in the list
    # column3: list length
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/TOP10PostingsWithTheirRanksFrom95KTrainingQueries"
    # on moa:
    # column0: qid
    # column1: query term
    # column2: docID
    # column3: rank in the list
    # column4: list length
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/TOP100PostingsWithRanksFrom95K_20140518"
    
    # on vidaserver1:
    # for gov2:
    # file format:
    # sample lines:
    # 1 orange 23097992 64132 283475
    # 1 revenue 23097992 120 667375
    # 1 commissioner 23097992 81 774380
    # column0: qid
    # column1: query term
    # column2: docID
    # column3: rank in the list
    # column4: list length
    inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/TOP100PostingsWithRanksFrom95K_20140518"
    
    # for clueweb09B:
    # file format:
    # column0: qid
    # column1: query term
    # column2: docID
    # column3: rank in the list
    # column4: list length
    # column5: impact score
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/clueweb09B/TOP100PostingsWithRanksANDScores_clueweb09B_trainingQueries_20140612"
    inputFileHandler = open(inputFileName,"r")
    
    for index,line in enumerate( inputFileHandler.readlines() ):
        # if index == 10:
        #     break
        lineElements = line.strip().split(" ")
        currentTerm = lineElements[1]
        rankForTheCurrentTerm = int(lineElements[3])
        if currentTerm not in termPostingListDict:
            termPostingListDict[currentTerm] = []
        termPostingListDict[currentTerm].append(rankForTheCurrentTerm)
    inputFileHandler.close()
    

    twoDimensionNumeratorTable = {}
    
    tempCounter1 = 0
    tempCounter3 = 0
    for term in termPostingListDict:
        tempCounter1 += len(termPostingListDict[term])
        termPostingListDict[term].sort()
        for currentRank in termPostingListDict[term]:
            tempCounter3 += 1
            if tempCounter3 % 1000000 == 0:
                print tempCounter3,"processed."
            # the value of the relRank
            currentRelRank = float(currentRank) / float(termWithTheirLengthOfListDict[term])

            '''
            # logic for stepGap to be 1/3
            initValue = 1.0
            currentUpperBound = initValue
            previousUpperBound = currentUpperBound
            # print "0",previousUpperBound
            for i in range(1,termWithTheirNumOfPieceDict[term]):
                currentUpperBound = previousUpperBound - previousUpperBound / 3
                # print str(i),currentUpperBound
                previousUpperBound = currentUpperBound
            
            # at this point, the currentUpperBound value is the thing we want
            # print "least upper bound:",currentUpperBound
            
            pieceID = termWithTheirNumOfPieceDict[term] - 1
            # This might be a problem in my c++ code, what do you think?
            if currentRelRank != 1:
                while currentRelRank > currentUpperBound:
                    currentUpperBound = currentUpperBound * 3 / 2
                    pieceID -= 1
            else:
                pieceID = 0
                currentUpperBound = 1
            currentClassBelongingTo = termWithTheirClassLabelBasedOnListLength[term]
            '''

            # logic for stepGap to be 1/2
            initValue = 1.0
            currentUpperBound = initValue
            previousUpperBound = currentUpperBound
            # print "0",previousUpperBound
            for i in range(1,termWithTheirNumOfPieceDict[term]):
                currentUpperBound = previousUpperBound - previousUpperBound / 2
                # print str(i),currentUpperBound
                previousUpperBound = currentUpperBound

            # at this point, the currentUpperBound value is the least upper bound
            # print "least upper bound:",currentUpperBound

            pieceID = termWithTheirNumOfPieceDict[term] - 1
            if currentRelRank != 1:
                while currentRelRank > currentUpperBound:
                    currentUpperBound = currentUpperBound * 2
                    pieceID -= 1
            else:
                pieceID = 0
                currentUpperBound = 1
            currentClassBelongingTo = termWithTheirClassLabelBasedOnListLength[term]


            # debug line
            # if currentClassBelongingTo == 68 and pieceID == 30:
            #    print term,currentClassBelongingTo,pieceID,currentUpperBound,currentRelRank,currentRank,termWithTheirLengthOfListDict[term]
            
            # print term,currentClassBelongingTo,pieceID,currentUpperBound,currentRelRank,currentRank,termWithTheirLengthOfListDict[term]      
            
            if currentClassBelongingTo not in twoDimensionNumeratorTable:
                twoDimensionNumeratorTable[currentClassBelongingTo] = {}

            if pieceID not in twoDimensionNumeratorTable[currentClassBelongingTo]:
                twoDimensionNumeratorTable[currentClassBelongingTo][pieceID] = 0

            twoDimensionNumeratorTable[currentClassBelongingTo][pieceID] += 1

    print "tempCounter1: ",tempCounter1
    # print "termPostingListDict['trec']: ",termPostingListDict['trec']

    '''
    # CURRENT version1
    tempCounter2 = 0
    for i in twoDimensionNumeratorTable:
        for j in twoDimensionNumeratorTable[i]:
            print i,j,twoDimensionNumeratorTable[i][j]
            tempCounter2 += twoDimensionNumeratorTable[i][j]
        print    
    print "tempCounter2: ",tempCounter2
    '''
 
    
    # working version
    # CURRENT version2
    tempCounter2 = 0
    for i in range(0,len(twoDimensionNumeratorTable)):
        print i,
        for j in range(0,classLabelWithTheirNumOfPiecesDict[i]):
            if j not in twoDimensionNumeratorTable[i]:
                print "0",
                # print i,j,"0"
            else:
                print twoDimensionNumeratorTable[i][j],
                # print i,j,twoDimensionNumeratorTable[i][j]
                tempCounter2 += twoDimensionNumeratorTable[i][j]
        print
    print "tempCounter2: ",tempCounter2
    

    print "Overall Processing Statistics:"
    print "inputAuxFileName0: ",inputAuxFileName0
    print "inputAuxFileName1: ",inputAuxFileName1
    print "inputFileName: ",inputFileName

def step1_build3DNumeratorTable():
    print "step1_build3DNumeratorTable() called."
    termWithTheirLengthOfListDict = {}
    termWithTheirNumOfPieceDict = {}
    termWithTheirClassLabelBasedOnListLength = {}
    classLabelWithTheirNumOfPiecesDict = {}

    # step0: assign the bin value to the corresponding docIDs
    # load the docID bins
    # It is a dictionary
    # key: docID
    # value: the corresponding bin value of a specific doc
    docIDAndDocHitFreqDict = {}
    ifn = "/san_share/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/TOP1000/AND/docIDAndTrecIDHitsCollection_notSorted_AND_fakeQueries_20140826"
    ifh = open(ifn,"r")
    l = ifh.readline()
    lineCounter = 1
    while l:
        le = l.strip().split(" ")
        docID = le[0]
        docHit = int(le[2])
        # do a classification using the docHit
        #bin0: 0
        #bin1: 1
        #bin2: 2
        #bin3: 3-4
        #bin4: 5-7
        #bin5: 8-12
        #bin6: 12-20
        #bin7: 20-40
        #bin8: 40-80
        #bin9: 80-160
        #bin10: 160-200
        #bin11: 200-500
        #bin12: 500-1000
        #bin13: > 1000
        if docHit == 1:
            docIDAndDocHitFreqDict[docID] = 1
        elif docHit == 2:
            docIDAndDocHitFreqDict[docID] = 2
        elif docHit >= 3 and docHit < 4:
            docIDAndDocHitFreqDict[docID] = 3
        elif docHit >= 5 and docHit < 7:
            docIDAndDocHitFreqDict[docID] = 4
        elif docHit >= 8 and docHit < 12:
            docIDAndDocHitFreqDict[docID] = 5
        elif docHit >= 12 and docHit < 20:
            docIDAndDocHitFreqDict[docID] = 6
        elif docHit >= 20 and docHit < 40:
            docIDAndDocHitFreqDict[docID] = 7
        elif docHit >= 40 and docHit < 80:
            docIDAndDocHitFreqDict[docID] = 8
        elif docHit >= 80 and docHit < 160:
            docIDAndDocHitFreqDict[docID] = 9
        elif docHit >= 160 and docHit < 200:
            docIDAndDocHitFreqDict[docID] = 10
        elif docHit >= 200 and docHit < 500:
            docIDAndDocHitFreqDict[docID] = 11
        elif docHit >= 500 and docHit < 1000:
            docIDAndDocHitFreqDict[docID] = 12
        else:
            docIDAndDocHitFreqDict[docID] = 13
        l = ifh.readline()
        lineCounter += 1
        if lineCounter % 100000 == 0:
            print lineCounter,"lines processed."
    print "len(docIDAndDocHitFreqDict):",len(docIDAndDocHitFreqDict)
    ifh.close()
    
    # stepGap to be 1/2
    # on moa:
    # inputAuxFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/secondFactorProbability/predefinedClassLabelUpperBoundOfRelRank20140102Night"
    # on vidaserver1:
    # gov2
    inputAuxFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/predefinedClassLabelUpperBoundOfRelRank20140102Night"
    # clueweb09B
    # inputAuxFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/clueweb09B/predefinedClassLabelLowerBoundOfListLength_clueweb09B_20140612"
    print "inputAuxFileName0:",inputAuxFileName0
    inputAuxFileHandler = open(inputAuxFileName0,"r")
    for line in inputAuxFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentClassLabel = int( lineElements[0] )
        # for gov2 format
        numOfPieces = int(lineElements[1])
        # for clueweb09B format
        # numOfPieces = int(lineElements[2])
        if currentClassLabel not in classLabelWithTheirNumOfPiecesDict:
             classLabelWithTheirNumOfPiecesDict[currentClassLabel] = numOfPieces
    inputAuxFileHandler.close()
    print "len(classLabelWithTheirNumOfPiecesDict): ",len(classLabelWithTheirNumOfPiecesDict)
    print "classLabelWithTheirNumOfPiecesDict[3]: ",classLabelWithTheirNumOfPiecesDict[3]

    # stepGap to be 1/2
    # for gov2
    # inputAuxFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/secondFactorProbability/termPieceInfoForQueryTerms_stepGap_2_OLD"
    # on vidaserver1
    # for gov2
    inputAuxFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/termPieceInfoForQueryTerms_stepGap_2_OLD"
    # for clueweb09B
    # inputAuxFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/clueweb09B/termPieceInfo_trainingQueryTerms_clueweb09B_20140613"
    inputAuxFileHandler = open(inputAuxFileName1,"r")
    
    for line in inputAuxFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # for gov2 format
        base = 0
        # for clueweb09B format
        # base = 1
        currentTerm = lineElements[base + 0]
        lengthOfTheList = int(lineElements[base + 1])
        classLabelBasedOnListLength = int(lineElements[base + 2])
        numOfPieces = int(lineElements[base + 3])
        # print currentTerm,lengthOfTheList,classLabelBasedOnListLength,numOfPieces
        if currentTerm not in termWithTheirLengthOfListDict:
            termWithTheirLengthOfListDict[currentTerm] = lengthOfTheList
            termWithTheirClassLabelBasedOnListLength[currentTerm] = classLabelBasedOnListLength
            termWithTheirNumOfPieceDict[currentTerm] = numOfPieces
    inputAuxFileHandler.close()
    print "len(termWithTheirLengthOfListDict): ",len(termWithTheirLengthOfListDict)
    print "termWithTheirLengthOfListDict['fawn']: ",termWithTheirLengthOfListDict['fawn']
    print "len(termWithTheirClassLabelBasedOnListLength): ",len(termWithTheirClassLabelBasedOnListLength)
    print "len(termWithTheirNumOfPieceDict): ",len(termWithTheirNumOfPieceDict)

    # key: term
    # value: a list containing all the postings
    termPostingListDict = {}
    # for gov2
    # the formats for the below two files are somehow different
    # on dodo:
    # column0: query term
    # column1: docID
    # column2: rank in the list
    # column3: list length
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/relRankFilesRelated/TOP10PostingsWithTheirRanksFrom95KTrainingQueries"
    # on moa:
    # column0: qid
    # column1: query term
    # column2: docID
    # column3: rank in the list
    # column4: list length
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/TOP100PostingsWithRanksFrom95K_20140518"
    
    # on vidaserver1:
    # for gov2:
    # file format:
    # sample lines:
    # 1 orange 23097992 64132 283475
    # 1 revenue 23097992 120 667375
    # 1 commissioner 23097992 81 774380
    # column0: qid 
    # column1: query term
    # column2: docID
    # column3: rank in the list
    # column4: list length
    inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/data/secondFactorProbability/gov2/TOP100PostingsWithRanksFrom95K_20140518"
    
    # for clueweb09B:
    # file format:
    # column0: qid
    # column1: query term
    # column2: docID
    # column3: rank in the list
    # column4: list length
    # column5: impact score
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/clueweb09B/TOP100PostingsWithRanksANDScores_clueweb09B_trainingQueries_20140612"
    inputFileHandler = open(inputFileName,"r")
    l = inputFileHandler.readline()
    lineCounter = 1
    # print "mark1"
    while l:
        # print "lineCounter:",lineCounter
        #if lineCounter == 10:
        #     break
        lineElements = l.strip().split(" ")
        currentTerm = lineElements[1]
        # need to put this into some bins
        docID = lineElements[2]
        rankForTheCurrentTerm = int(lineElements[3])
        if currentTerm not in termPostingListDict:
            termPostingListDict[currentTerm] = []
        termPostingListDict[currentTerm].append((docID,rankForTheCurrentTerm))
        l = inputFileHandler.readline()
        lineCounter += 1
        if lineCounter % 100000 == 0:
            print lineCounter,"lines processed."
    inputFileHandler.close()
    

    threeDimensionNumeratorTable = {}
    
    tempCounter1 = 0
    tempCounter3 = 0
    for term in termPostingListDict:
        tempCounter1 += len(termPostingListDict[term])
        termPostingListDict[term].sort()
        for (docID,currentRank) in termPostingListDict[term]:
            tempCounter3 += 1
            if tempCounter3 % 1000000 == 0:
                print tempCounter3,"processed."
            # the value of the relRank
            currentRelRank = float(currentRank) / float(termWithTheirLengthOfListDict[term])

            # logic for stepGap to be 1/2
            initValue = 1.0
            currentUpperBound = initValue
            previousUpperBound = currentUpperBound
            # print "0",previousUpperBound
            for i in range(1,termWithTheirNumOfPieceDict[term]):
                currentUpperBound = previousUpperBound - previousUpperBound / 2
                # print str(i),currentUpperBound
                previousUpperBound = currentUpperBound

            # at this point, the currentUpperBound value is the least upper bound
            # print "least upper bound:",currentUpperBound

            pieceID = termWithTheirNumOfPieceDict[term] - 1
            if currentRelRank != 1:
                while currentRelRank > currentUpperBound:
                    currentUpperBound = currentUpperBound * 2
                    pieceID -= 1
            else:
                pieceID = 0
                currentUpperBound = 1
            currentClassBelongingTo = termWithTheirClassLabelBasedOnListLength[term]

            # third dimension
            if docID not in docIDAndDocHitFreqDict:
                binValue = 0
            else:
                binValue = docIDAndDocHitFreqDict[docID]

            # debug line
            # if currentClassBelongingTo == 68 and pieceID == 30:
            #    print term,currentClassBelongingTo,pieceID,currentUpperBound,currentRelRank,currentRank,termWithTheirLengthOfListDict[term]
            
            # print term,currentClassBelongingTo,pieceID,currentUpperBound,currentRelRank,currentRank,termWithTheirLengthOfListDict[term]      
            
            if currentClassBelongingTo not in threeDimensionNumeratorTable:
                threeDimensionNumeratorTable[currentClassBelongingTo] = {}

            if pieceID not in threeDimensionNumeratorTable[currentClassBelongingTo]:
                threeDimensionNumeratorTable[currentClassBelongingTo][pieceID] = {}
            
            if binValue not in threeDimensionNumeratorTable[currentClassBelongingTo][pieceID]:
                threeDimensionNumeratorTable[currentClassBelongingTo][pieceID][binValue] = 0

            threeDimensionNumeratorTable[currentClassBelongingTo][pieceID][binValue] += 1
    print "tempCounter1: ",tempCounter1
    
    # let's change the presentation for the data somehow
    # modified version for 3 dimensions
    tempCounter2 = 0
    for i in threeDimensionNumeratorTable:
        # print i
        for j in threeDimensionNumeratorTable[i]:
            # print j
            for k in threeDimensionNumeratorTable[i][j]:
                print i,j,k,threeDimensionNumeratorTable[i][j][k]
                tempCounter2 += threeDimensionNumeratorTable[i][j][k]
    print "tempCounter2:",tempCounter2
    
    
    print "Overall Processing Statistics:"
    print "ifn: ",ifn
    print "inputAuxFileName0: ",inputAuxFileName0
    print "inputAuxFileName1: ",inputAuxFileName1
    print "inputFileName: ",inputFileName

print "Program Begins..."
# step1_build2DNumeratorTable()
step1_build3DNumeratorTable()
print "Program Ends."
